package com.coderfans.datacsdn.content;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import us.codecraft.xsoup.Xsoup;

import java.io.File;

/**
 * Created by 58-daojia on 2018-3-16.
 *
 * @author: fanchunshuai
 * @version: V1.0
 * @Desc:  读取博客内容重新生成txt文件并进行相似性计算
 * @Copyright (c) 2018 58到家-版权所有
 */
public class ReadOneHtmlContent {

    public static void main(String[] args) {
        Document document = null;
        try {
            String newBasicFilePath = "E:\\spiderFile\\csdn_content";
            String oldFilePath = "E:\\spiderFile\\csdn";

            File file = new File("E:\\spiderFile\\csdn");

            long startTime = System.currentTimeMillis();

            File [] fileList = file.listFiles();
            for (File userFile : fileList){
                if(userFile.isDirectory()){
                    File[] htmlFileList = userFile.listFiles();
                    for(File htmlFile : htmlFileList){
                        document = Jsoup.parse(htmlFile, "UTF-8");
                        //过滤无用文件
                        if( (htmlFile.length()/1024) <=2){
                            continue;
                        }
                        String  xpathValue = "//*[@id=\"article_content\"]";
                        String content = Xsoup.compile(xpathValue).evaluate(document).get();
                        //过滤无法提取主内容的文件
                        if(StringUtils.isBlank(content)){
                            continue;
                        }

                        String filePath = htmlFile.getAbsolutePath().replace(oldFilePath,"").replace("'","").replace(".html","");

                        String newFilePath = newBasicFilePath + filePath + ".txt";
                        System.out.println(newFilePath);
                        document = Jsoup.parse(content, "UTF-8");
                        document.text();
                        File newFile = new File(newFilePath);

                        FileUtils.writeStringToFile(newFile,document.text());

                    }
                }

            }

            long endTime = System.currentTimeMillis();
            System.out.println("endTime - startTime = "+(endTime - startTime) / 1000+"s");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
